{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Elibitility Trace Actor-Critic to Play Acrobot-v1\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import copy\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "torch.manual_seed(0)\n",
    "import torch.autograd as autograd\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.optim as optim\n",
    "import torch.distributions as distributions\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14:21:37 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "14:21:37 [INFO] action_space: Discrete(3)\n",
      "14:21:37 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "14:21:37 [INFO] reward_range: (-inf, inf)\n",
      "14:21:37 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "14:21:37 [INFO] _max_episode_steps: 500\n",
      "14:21:37 [INFO] _elapsed_steps: None\n",
      "14:21:37 [INFO] id: Acrobot-v1\n",
      "14:21:37 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv\n",
      "14:21:37 [INFO] reward_threshold: -100.0\n",
      "14:21:37 [INFO] nondeterministic: False\n",
      "14:21:37 [INFO] max_episode_steps: 500\n",
      "14:21:37 [INFO] _kwargs: {}\n",
      "14:21:37 [INFO] _env_name: Acrobot\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ElibilityTraceActorCriticAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "        self.actor_lambda = 0.9\n",
    "        self.critic_lambda = 0.9\n",
    "\n",
    "        self.actor_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,],\n",
    "                output_size=env.action_space.n, output_activator=nn.Softmax(1))\n",
    "        self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 0.0001)\n",
    "        self.actor_trace = copy.deepcopy(self.actor_net)\n",
    "\n",
    "        self.critic_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,], output_size=self.action_n)\n",
    "        self.critic_optimizer = optim.Adam(self.critic_net.parameters(), 0.0002)\n",
    "        self.critic_loss = nn.MSELoss()\n",
    "        self.critic_trace = copy.deepcopy(self.critic_net)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "\n",
    "            def weights_init(m):\n",
    "                if isinstance(m, nn.Linear):\n",
    "                    init.zeros_(m.weight)\n",
    "                    init.zeros_(m.bias)\n",
    "            self.actor_trace.apply(weights_init)\n",
    "            self.critic_trace.apply(weights_init)\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)\n",
    "        prob_tensor = self.actor_net(state_tensor)\n",
    "        action_tensor = distributions.Categorical(prob_tensor).sample()\n",
    "        action = action_tensor.numpy()[0]\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, target_weight, evaluate_weight):\n",
    "        for target_param, evaluate_param in zip(\n",
    "                target_net.parameters(), evaluate_net.parameters()):\n",
    "            target_param.data.copy_(evaluate_weight * evaluate_param.data\n",
    "                    + target_weight * target_param.data)\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, next_action = \\\n",
    "                self.trajectory[-8:]\n",
    "        state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "        next_state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "\n",
    "        pred_tensor = self.critic_net(state_tensor)\n",
    "        pred = pred_tensor.detach().numpy()[0, 0]\n",
    "        next_v_tesnor = self.critic_net(next_state_tensor)\n",
    "        next_v = next_v_tesnor.detach().numpy()[0, 0]\n",
    "        target = reward + (1. - done) * self.gamma * next_v\n",
    "        td_error = target - pred\n",
    "\n",
    "        # train actor\n",
    "        pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "        logpi_tensor = torch.log(torch.clamp(pi_tensor, 1e-6, 1.))\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        logpi_tensor.backward(retain_graph=True)\n",
    "        for param, trace in zip(self.actor_net.parameters(), self.actor_trace.parameters()):\n",
    "            trace.data.copy_(self.gamma * self.actor_lambda * trace.data + self.discount * param.grad)\n",
    "            param.grad.copy_(-td_error * trace)\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "        # train critic\n",
    "        v_tensor = self.critic_net(state_tensor)[0, 0]\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        v_tensor.backward()\n",
    "        for param, trace in zip(self.critic_net.parameters(), self.critic_trace.parameters()):\n",
    "            trace.data.copy_(self.gamma * self.critic_lambda * trace.data + param.grad)\n",
    "            param.grad.copy_(-td_error * trace)\n",
    "        self.critic_optimizer.step()\n",
    "\n",
    "\n",
    "agent = ElibilityTraceActorCriticAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14:21:37 [INFO] ==== train ====\n",
      "14:21:40 [DEBUG] train episode 0: reward = -500.00, steps = 500\n",
      "14:21:43 [DEBUG] train episode 1: reward = -500.00, steps = 500\n",
      "14:21:45 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "14:21:47 [DEBUG] train episode 3: reward = -346.00, steps = 347\n",
      "14:21:48 [DEBUG] train episode 4: reward = -149.00, steps = 150\n",
      "14:21:50 [DEBUG] train episode 5: reward = -311.00, steps = 312\n",
      "14:21:51 [DEBUG] train episode 6: reward = -323.00, steps = 324\n",
      "14:21:52 [DEBUG] train episode 7: reward = -184.00, steps = 185\n",
      "14:21:53 [DEBUG] train episode 8: reward = -143.00, steps = 144\n",
      "14:21:54 [DEBUG] train episode 9: reward = -225.00, steps = 226\n",
      "14:21:56 [DEBUG] train episode 10: reward = -207.00, steps = 208\n",
      "14:21:56 [DEBUG] train episode 11: reward = -171.00, steps = 172\n",
      "14:21:57 [DEBUG] train episode 12: reward = -123.00, steps = 124\n",
      "14:21:58 [DEBUG] train episode 13: reward = -208.00, steps = 209\n",
      "14:21:59 [DEBUG] train episode 14: reward = -188.00, steps = 189\n",
      "14:22:00 [DEBUG] train episode 15: reward = -209.00, steps = 210\n",
      "14:22:01 [DEBUG] train episode 16: reward = -123.00, steps = 124\n",
      "14:22:02 [DEBUG] train episode 17: reward = -158.00, steps = 159\n",
      "14:22:03 [DEBUG] train episode 18: reward = -143.00, steps = 144\n",
      "14:22:04 [DEBUG] train episode 19: reward = -204.00, steps = 205\n",
      "14:22:05 [DEBUG] train episode 20: reward = -165.00, steps = 166\n",
      "14:22:06 [DEBUG] train episode 21: reward = -148.00, steps = 149\n",
      "14:22:07 [DEBUG] train episode 22: reward = -145.00, steps = 146\n",
      "14:22:08 [DEBUG] train episode 23: reward = -115.00, steps = 116\n",
      "14:22:08 [DEBUG] train episode 24: reward = -159.00, steps = 160\n",
      "14:22:09 [DEBUG] train episode 25: reward = -135.00, steps = 136\n",
      "14:22:10 [DEBUG] train episode 26: reward = -117.00, steps = 118\n",
      "14:22:11 [DEBUG] train episode 27: reward = -137.00, steps = 138\n",
      "14:22:11 [DEBUG] train episode 28: reward = -129.00, steps = 130\n",
      "14:22:12 [DEBUG] train episode 29: reward = -102.00, steps = 103\n",
      "14:22:13 [DEBUG] train episode 30: reward = -119.00, steps = 120\n",
      "14:22:13 [DEBUG] train episode 31: reward = -99.00, steps = 100\n",
      "14:22:14 [DEBUG] train episode 32: reward = -122.00, steps = 123\n",
      "14:22:15 [DEBUG] train episode 33: reward = -134.00, steps = 135\n",
      "14:22:15 [DEBUG] train episode 34: reward = -108.00, steps = 109\n",
      "14:22:16 [DEBUG] train episode 35: reward = -131.00, steps = 132\n",
      "14:22:16 [INFO] ==== test ====\n",
      "14:22:16 [DEBUG] test episode 0: reward = -130.00, steps = 131\n",
      "14:22:16 [DEBUG] test episode 1: reward = -226.00, steps = 227\n",
      "14:22:16 [DEBUG] test episode 2: reward = -133.00, steps = 134\n",
      "14:22:17 [DEBUG] test episode 3: reward = -219.00, steps = 220\n",
      "14:22:17 [DEBUG] test episode 4: reward = -189.00, steps = 190\n",
      "14:22:17 [DEBUG] test episode 5: reward = -106.00, steps = 107\n",
      "14:22:17 [DEBUG] test episode 6: reward = -107.00, steps = 108\n",
      "14:22:17 [DEBUG] test episode 7: reward = -111.00, steps = 112\n",
      "14:22:17 [DEBUG] test episode 8: reward = -162.00, steps = 163\n",
      "14:22:17 [DEBUG] test episode 9: reward = -105.00, steps = 106\n",
      "14:22:17 [DEBUG] test episode 10: reward = -133.00, steps = 134\n",
      "14:22:17 [DEBUG] test episode 11: reward = -123.00, steps = 124\n",
      "14:22:17 [DEBUG] test episode 12: reward = -96.00, steps = 97\n",
      "14:22:18 [DEBUG] test episode 13: reward = -118.00, steps = 119\n",
      "14:22:18 [DEBUG] test episode 14: reward = -105.00, steps = 106\n",
      "14:22:18 [DEBUG] test episode 15: reward = -126.00, steps = 127\n",
      "14:22:18 [DEBUG] test episode 16: reward = -182.00, steps = 183\n",
      "14:22:18 [DEBUG] test episode 17: reward = -119.00, steps = 120\n",
      "14:22:18 [DEBUG] test episode 18: reward = -137.00, steps = 138\n",
      "14:22:18 [DEBUG] test episode 19: reward = -133.00, steps = 134\n",
      "14:22:18 [DEBUG] test episode 20: reward = -287.00, steps = 288\n",
      "14:22:19 [DEBUG] test episode 21: reward = -169.00, steps = 170\n",
      "14:22:19 [DEBUG] test episode 22: reward = -217.00, steps = 218\n",
      "14:22:19 [DEBUG] test episode 23: reward = -272.00, steps = 273\n",
      "14:22:19 [DEBUG] test episode 24: reward = -105.00, steps = 106\n",
      "14:22:19 [DEBUG] test episode 25: reward = -223.00, steps = 224\n",
      "14:22:19 [DEBUG] test episode 26: reward = -142.00, steps = 143\n",
      "14:22:19 [DEBUG] test episode 27: reward = -148.00, steps = 149\n",
      "14:22:20 [DEBUG] test episode 28: reward = -158.00, steps = 159\n",
      "14:22:20 [DEBUG] test episode 29: reward = -144.00, steps = 145\n",
      "14:22:20 [DEBUG] test episode 30: reward = -153.00, steps = 154\n",
      "14:22:20 [DEBUG] test episode 31: reward = -126.00, steps = 127\n",
      "14:22:20 [DEBUG] test episode 32: reward = -210.00, steps = 211\n",
      "14:22:20 [DEBUG] test episode 33: reward = -136.00, steps = 137\n",
      "14:22:20 [DEBUG] test episode 34: reward = -140.00, steps = 141\n",
      "14:22:20 [DEBUG] test episode 35: reward = -123.00, steps = 124\n",
      "14:22:20 [DEBUG] test episode 36: reward = -107.00, steps = 108\n",
      "14:22:21 [DEBUG] test episode 37: reward = -139.00, steps = 140\n",
      "14:22:21 [DEBUG] test episode 38: reward = -118.00, steps = 119\n",
      "14:22:21 [DEBUG] test episode 39: reward = -118.00, steps = 119\n",
      "14:22:21 [DEBUG] test episode 40: reward = -118.00, steps = 119\n",
      "14:22:21 [DEBUG] test episode 41: reward = -250.00, steps = 251\n",
      "14:22:21 [DEBUG] test episode 42: reward = -140.00, steps = 141\n",
      "14:22:21 [DEBUG] test episode 43: reward = -160.00, steps = 161\n",
      "14:22:21 [DEBUG] test episode 44: reward = -115.00, steps = 116\n",
      "14:22:21 [DEBUG] test episode 45: reward = -148.00, steps = 149\n",
      "14:22:22 [DEBUG] test episode 46: reward = -128.00, steps = 129\n",
      "14:22:22 [DEBUG] test episode 47: reward = -206.00, steps = 207\n",
      "14:22:22 [DEBUG] test episode 48: reward = -172.00, steps = 173\n",
      "14:22:22 [DEBUG] test episode 49: reward = -152.00, steps = 153\n",
      "14:22:22 [DEBUG] test episode 50: reward = -142.00, steps = 143\n",
      "14:22:22 [DEBUG] test episode 51: reward = -178.00, steps = 179\n",
      "14:22:22 [DEBUG] test episode 52: reward = -84.00, steps = 85\n",
      "14:22:22 [DEBUG] test episode 53: reward = -115.00, steps = 116\n",
      "14:22:23 [DEBUG] test episode 54: reward = -229.00, steps = 230\n",
      "14:22:23 [DEBUG] test episode 55: reward = -226.00, steps = 227\n",
      "14:22:23 [DEBUG] test episode 56: reward = -104.00, steps = 105\n",
      "14:22:23 [DEBUG] test episode 57: reward = -172.00, steps = 173\n",
      "14:22:23 [DEBUG] test episode 58: reward = -115.00, steps = 116\n",
      "14:22:23 [DEBUG] test episode 59: reward = -141.00, steps = 142\n",
      "14:22:23 [DEBUG] test episode 60: reward = -139.00, steps = 140\n",
      "14:22:23 [DEBUG] test episode 61: reward = -148.00, steps = 149\n",
      "14:22:24 [DEBUG] test episode 62: reward = -256.00, steps = 257\n",
      "14:22:24 [DEBUG] test episode 63: reward = -172.00, steps = 173\n",
      "14:22:24 [DEBUG] test episode 64: reward = -161.00, steps = 162\n",
      "14:22:24 [DEBUG] test episode 65: reward = -134.00, steps = 135\n",
      "14:22:24 [DEBUG] test episode 66: reward = -201.00, steps = 202\n",
      "14:22:24 [DEBUG] test episode 67: reward = -117.00, steps = 118\n",
      "14:22:24 [DEBUG] test episode 68: reward = -125.00, steps = 126\n",
      "14:22:24 [DEBUG] test episode 69: reward = -122.00, steps = 123\n",
      "14:22:24 [DEBUG] test episode 70: reward = -197.00, steps = 198\n",
      "14:22:25 [DEBUG] test episode 71: reward = -145.00, steps = 146\n",
      "14:22:25 [DEBUG] test episode 72: reward = -124.00, steps = 125\n",
      "14:22:25 [DEBUG] test episode 73: reward = -147.00, steps = 148\n",
      "14:22:25 [DEBUG] test episode 74: reward = -134.00, steps = 135\n",
      "14:22:25 [DEBUG] test episode 75: reward = -129.00, steps = 130\n",
      "14:22:25 [DEBUG] test episode 76: reward = -107.00, steps = 108\n",
      "14:22:25 [DEBUG] test episode 77: reward = -171.00, steps = 172\n",
      "14:22:25 [DEBUG] test episode 78: reward = -82.00, steps = 83\n",
      "14:22:25 [DEBUG] test episode 79: reward = -117.00, steps = 118\n",
      "14:22:26 [DEBUG] test episode 80: reward = -131.00, steps = 132\n",
      "14:22:26 [DEBUG] test episode 81: reward = -145.00, steps = 146\n",
      "14:22:26 [DEBUG] test episode 82: reward = -111.00, steps = 112\n",
      "14:22:26 [DEBUG] test episode 83: reward = -170.00, steps = 171\n",
      "14:22:26 [DEBUG] test episode 84: reward = -161.00, steps = 162\n",
      "14:22:26 [DEBUG] test episode 85: reward = -109.00, steps = 110\n",
      "14:22:26 [DEBUG] test episode 86: reward = -111.00, steps = 112\n",
      "14:22:26 [DEBUG] test episode 87: reward = -98.00, steps = 99\n",
      "14:22:26 [DEBUG] test episode 88: reward = -142.00, steps = 143\n",
      "14:22:26 [DEBUG] test episode 89: reward = -155.00, steps = 156\n",
      "14:22:27 [DEBUG] test episode 90: reward = -85.00, steps = 86\n",
      "14:22:27 [DEBUG] test episode 91: reward = -101.00, steps = 102\n",
      "14:22:27 [DEBUG] test episode 92: reward = -165.00, steps = 166\n",
      "14:22:27 [DEBUG] test episode 93: reward = -125.00, steps = 126\n",
      "14:22:27 [DEBUG] test episode 94: reward = -114.00, steps = 115\n",
      "14:22:27 [DEBUG] test episode 95: reward = -145.00, steps = 146\n",
      "14:22:27 [DEBUG] test episode 96: reward = -131.00, steps = 132\n",
      "14:22:27 [DEBUG] test episode 97: reward = -126.00, steps = 127\n",
      "14:22:27 [DEBUG] test episode 98: reward = -173.00, steps = 174\n",
      "14:22:27 [DEBUG] test episode 99: reward = -112.00, steps = 113\n",
      "14:22:27 [INFO] average episode reward = -146.28 ± 41.48\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3aklEQVR4nO3deXzU5bX48c/JvidAAgkkEJawyRIhIqhQF1q1tWK19WJVUKuo1Vtv29v2ql1tvfXX3tpbe6sWrQvWjQoudalKXasghC2A7GtWkpB9myzz/P6Y78AAM8msSSY579drXky+31keBj155vme5xwxxqCUUmpwiejrASillOp9GvyVUmoQ0uCvlFKDkAZ/pZQahDT4K6XUIBTV1wPwVnp6usnNze3rYSilVFjZuHFjtTEm49TjYRP8c3NzKSws7OthKKVUWBGRw+6O67KPUkoNQhr8lVJqENLgr5RSg5AGf6WUGoQ0+Cul1CCkwV8ppQahgIK/iHxDRHaIiF1ECk45d7eI7BOR3SJyscvx2SKyzTr3kIhIIGNQSinlu0Bn/tuBK4GPXA+KyFRgMXAGcAnwsIhEWqcfAZYBedbtkgDHoJRSQdNs6+T59Udobe/q66GEVEDB3xiz0xiz282pRcALxhibMeYgsA+YIyJZQIoxZq1xNBJYAVwRyBiUUsrJGEMgPUo6uuzc/uwm7l69jbtXFwX0Wv1dqNb8RwHFLj+XWMdGWfdPPe6WiCwTkUIRKayqqgrJQJVSA8c9L2/j0j98THl9q8/PNcZwz+ptfLSnivl56byypYxnPzsSglH2Dz0GfxFZIyLb3dwWdfc0N8dMN8fdMsYsN8YUGGMKMjJOK02hlFLHvfv5UZ5fX8zuo41c/ee1FNe0+PT8P/xzL3/bWMJ3Lsrj6RvncP6kDO77++cUldSFZsB9rMfgb4xZaIyZ5ub2ajdPKwFyXH7OBsqs49lujiullN/qWzu49+VtTM5M5qXb5tHQ2snVf17Lwepmr56/ckMx/7tmL1+fnc13F+YRESH8/up8MpJjuf2vm6hraQ/JuLvsfbesFKpln9eAxSISKyJjcVzYXW+MKQcaRWSuleWzBOjul4hSSvXov9/YSXWTjd98fQazxwzl+Vvm0t5p5+o/r2XP0cZun/vB7krufnkb8/PS+fWV03EmIA5JjOFP186isrGN763cij3IgXrzkVpm/fJdVhYW9/zgEAg01fNrIlICzAPeEJG3AYwxO4CVwOfAP4A7jDHOS+e3A4/juAi8H3grkDEopQa3f+2t5sXCYpYtGM+M7DQApo5M4cVb5yLA4uXr2FFW7/a520vruePZTUwakczD184iOvLkkJifk8ZPLpvKe7sqeeTD/UEbc2t7F99fuZX61g5+/Mp2tpe6H18oBZrt87IxJtsYE2uMGWGMudjl3P3GmPHGmEnGmLdcjhday0bjjTF3moF8OV0pFVLNtk7+a3UR49IT+Y+FeSedmzA8mZW3ziM+OpJrlq9jS3HdSeeLa1q48akNpCXE8OSNZ5EcF+32Pa6fO4avzhzJ797Zzaf7q4My7t++vZsD1c388ZozGZYYw7ef3UR9a0dQXttbusNXqUFs1cYSPtwTvpl0v317N6V1rfy/r88gLjrytPO56Ym8eOtc0hJiuO7xz1h/sAaAupZ2bnhyPbaOLp668SxGpMR5fA8R4YErpzM2PZHvPL+Zow1tAY153YFjPPHJQZbOc/xS+b9vzqKsrpXvr9zaq6mlGvyVGqR2ljfwg5e2cs/qbSG/8PjqllJWbyrp+YE+KDxUw9NrD7F0Xi5n5Q71+LjsIQmsvHUew1NiWfrEet7bdZRlKzZSXNPKY0sKyBuR3ON7JcZG8eh1s2m2dXHnc5vo6LL7NeYmWyc/eGkrucMS+NGlkwGYPWYI93x5Cmt2HmX5Rwf8el1/aPBXahAyxvDz13YAUFrXykd7Qzf7X7H2EHe9sIXvrdzKy5uD8wugraOLH64qYlRaPD+4eFKPj89MjePFZfMYMyyBm54qZP2hGh78t5mcPW6Y1++ZNyKZB66azoZDtfz2bXd7W3v232/upKS2lf/5xkwSYk40Urzx3Fy+Mj2L37y9m88OHPPrtX2lwV+pQej1onI+O1jDTy6bSnpSDM+FaDPTc58d4aev7mDhlBHMGzeMH75UxCf7Al83/8M/93KgqplfXzmdxFjvutFmJMfy/C1zWThlBL+6YhqXzRjp8/suyh/FdXNHs/yjA/xje4VPz/1wTxXPfXaEW+aPo+CUbyoiwgNXTWfM0ATufH4zlY2BLS15Q4O/6lOflzXw4Du7+8U2+rd3VPD8+oG7o9Oppb2T/35zJ1OzUlgyL5evz87hvV2VVNQHN+Cs3FDMPS9v44JJGfzp2jN59PrZjEtP4rZnNrKzvMHv191WUs/yjw7wbwU5zM/zbfPnkMQYHl9awHVzx/j9/j+5bCozslO564XNPP7xAa9SQOtbO/jRS0VMGJ7E97440e1jkuOiefi6WTS2dfCd5zfT6efSkrc0+Ks+9fs1e3jovX2U1Pq+HT/Y/vjeXn7yynafd4aGm4ff3095fRu/WHQGkRHC4rNy6LKboOabr9pYwo9WFzE/L51HrptNbFQkqfHRPHnjWSTGRnHjkxsoq/P937y9084PXtpKelIM93xlStDG64vYqEieuOEszpuQzq/e2Mnix9b1+N/ML/6+g6omGw9ePdPthWmnyZkp/PfXprPuQA2/e3dPsId+Eg3+qs/UNLfz/q5KAIpKej/P2VVbRxe7yhvptBv++N7ePh1LKB0+1szyjw5wRf7I4xdJc9MTOW9COi+sPxKUC7+vbinlBy9tZd64YTy2pOCkYDcyLZ4nbzyLZlsnNzy53uf0xkc/3M+uikZ+dcV0UuPdp2b2hvSkWB5fWsBvvj6Dz8sauPh/P+K5z464/Qb7zo4KVm8q5Y7zT+xD6M6Vs7K5Zs5oHvlgP2s+PxqC0Tto8Fd95u9by+i0GyKEPq+fsqvCEfjHpSeyalMph495VxYg3Pzy9Z1ERQr/denJs+Zvnj2asvo2Pgow7fONonK+t3IrBblDeXxpgdtZ7pSsFB69fjYHqpq59ZlCbJ09l062dXbx4oYj/PG9vVw+cyRfnDoioHEGg4hwdUEOb393Afk5adzz8jaWPrnhpOWzmuZ27nl5G1OzUrjzwrxuXu1kP/vqVKaNSuF7K7eE7JuoBn/VZ1ZvKmFqVgrTR6WytY+Dv/OXz++unklUhPDH9/b16XhC4YPdlazZeZR/vzCPzNST89oXThlBelJMQFUs395RwV0vbObMnDSevOGsk7JZTnXuhHR+8/UZrDtQww/+VuRx3byupZ0/vb+P8/7f+/xo1TYmZ6bws69O9XuMoTAqLZ6/futs7lt0BhsO1vCl33/Iy5tLMMbw41e2Ud/awYP/NpOYKO/DbVx0JI9cOxuA25/dSFtH8HsLaPBXfWJfZSNbS+q5ctYoZuaksb20Iei1U3xRVFJPelIM+TlpXHv2GF7eXMohL4uChYP2Tjv3/f1zxqYnctN5uaedj4mK4BsFOby366hf5ZD/ufModz63ienZqcfX9Xty5axsfnDxJF7bWsZvTkmdLK5p4eev7eCcB97jt2/vZkpWCn/91tm8due5DEuK9Xl8oRYRISyZl8ubd80nb0Qy331xK197+FPe3FbBfyycyOTMFJ9fM2doAg9enU9sVCRNts7gjznor6iUF1ZvKiUyQrg8fyQzstNosnVyoLqpz8ZTVFLH9FGpiAi3nT+O6EjhoQG09v/Upwc5UN3MTy+bSmyU+wuO15w1GruBlRt8y8Vfu/8Yt/91E1OyUnj6pjkeyyS48+3zx3Pt2aN59MP9PLP2EFuL67jjuU184bfv8+xnh7l0WhZv3TWfFTfN4by8dPp719ex6YmsvHUed186mc/LGsjPSePWBeP8fr2FU0fwt1vnkR6CX3jeJcgqFUR2u+HlzaUsyEtneHIcM7NTAdhSXM+E4T3vtgy2Zlsn+yqbuHRaFgDDk+O47uwxPPHJQe68YALjMpJ6fUzBVNnQxh/W7OWiycO5YPJwj48bPSyB+XnpvLjhCHdeOIHIiJ4DbVWjje+8sJmcofE8c9PZpPgQ+MGxbv6Ly8/gaEMbP3nVseksOTaKWxaM48Zzxp62PBUOIiOEW78wnkX5o0iKiyIqMrA5doQX/w5+vW5IXlWpbqw9cIzy+jaunOVo7TAuI4nEmMg+u+i7o6wBu4EZ1i8hgFu/MJ6YqAge+mf4z/4feGsXHV2Gn1zW81r5N+c4Lvx+sLuyx8fa7Ybv/81RmfJP184iNcG/7JuoyAgeuuZMFp+Vw4+/MoVP776Quy+dEpaB31VmahxJXm5A6wsa/FWvW7WphOTYqOMZG5ERwrRRqWzto3RP5y+d6S7BPyM5liXzcnltaxn7KvtuOSpQGw/XsHpzKTfPH0tuemKPj184dQTpSbFebXZ77OMDfLSnip9eNtWvNW1XCTFRPHDVDG6eP86nZSPlPw3+g1BxTQs3PLmeqkZbr793s62Tf2yv4Cszsk5KA5yZk8bOsgbaO0O7q9GdopJ6slLjGJ588kxz2YJxxEZF+jT7/9feapatKKS2OTSdn3zRZTf87LUdZKbEcccFE7x6TnRkBFcXZPPerspuN2FtPuKob3PptEyuPXt0sIaselGgzVy+ISI7RMQuIgUux3NFpFVEtli3R13OzRaRbSKyT0Qekv5+BWcAeuzjA3ywu8qrr/bB9vaOClrau44v+TjNyE6lvcvO7oruuy6FwrbS+pOWfJzSk2JZcs4Y/l5Uxt4eukGBo47N0ifX887nR3l7h291X4Kh2dbJluI6VhYW86vXP+eax9axvbSBu7882ev6NwDXzBmNAV7c4H7Hb31rB//+/GZGpMTxwFUz+v1FWOVeoAtS24ErgT+7ObffGJPv5vgjwDJgHfAmcAnazavXNLR18NJGRzbHluI6vlGQ08Mzgmv1plJyhsZTMGbIScdnWjsft5bUnbT8Emr1rR0crG7m67Oz3Z6/dcF4nll7mD/8cy//981Zbh/TZTf8+s2dPP6vg3xhYgZ7jjby/u5KFs8J3Yy4y254e0cF20rr2VPRyJ7KRoprTszUY6MimDA8iW+fP57LZ/pWwCxnaALz8zJYWVjMv1844aQLlsYY7lm9jfL6Nv5227w+3WWrAhNQ8DfG7AS8/s0vIllAijFmrfXzCuAKBkHw77Ibvv3sRm6eP67b2uOh9rfCElrauxiZGndaZ6NQK69v5ZP91XznwrzTMhiyh8QzNDHGWn/3v+iWr5zt89zN/AGGJsaw9JxcHv1wP9852sjEU2q/N9s6ueuFLazZeZSl88bwk8um8tPXdvDq5lLaO+0+bezxVmeXnf/821Ze2VJGdKQwLj2J/JwhXD07h4mZyUwckczooQleZet48s05Odz21018sLuKhS67aZ9fX8wb28r50SWTmTV6SDevoPq7UF6KHisim4EG4MfGmI+BUYBrEnGJdcwtEVmG41sCo0eH97piVaONt3ccZXxGUp8F/y674elPD1EwZghnjxvKox8eoLW9i/gYz4WmgumVzWUYA1fOOv2fXESYkZ3K1uLevejr3Fk8fZTnbxvL5o9jxaeH+MOavfzp2hOz//L6Vr71VCG7Khr4+VencsO5YwG4YNJwnvvsCIWHajhnQnpQx9veaeeuFzbz1vYK/vNLE7n1C+NP6zsbDBdNGUFGcizPrT9yPPjvrmjkF3/fwfy89IBy11X/0ON/NSKyRkS2u7kt6uZp5cBoY8yZwPeA50QkBXA3FfG4rdMYs9wYU2CMKcjI8K10a39T3eS4uBrssrm++GB3JUdqWrjh3Fzyc4bQZTds99DYOtiMMazeVELBmCGMGeY+62RGdhp7KxtpaQ/+bkZPtpXUM2ZYAmkJMR4fMyQxhhvPHcsb28rZVeEoRby9tJ4r/vQJh48185elZx0P/ADnjB9GTGQE7wf5mkpbRxe3/3Ujb22v4CeXTeXOC/NCEvjhxIXfD3ZXUlrXSmu7o4NVclw0D16dH7Lcc9V7evwvxxiz0Gq4furt1W6eYzPGHLPubwT2AxNxzPRdF1ezgbLA/grhocoK/uV9GPyf+vQQmSlxXHxGJvk5aQBsOVLXK++9vbSBvZVNp13odTUzOxW7cTy2txSV1Hc763e6ef5YkmOj+MOavbyzo4JvPLqWSBFeuv2c0zZOJcZGcfa4oby/O3jdsVraO7n56UL+uauSX10xjW+dN7bnJwVo8VknLvze9/oO9lY28ft/m0lGcv8rr6B8F5Jpg4hkiEikdX8ckAccMMaUA40iMtfK8lkCePwlMpBUW2mVgTZ/9tfeo418vLea6+eNIToygozkWLKHxPfauv+qTSXEREXwlelZHh/jLHfbW5u9qptslNa1Hr/Y3J20hBhuPDeXt7ZXcOtfNzJxRBKv3HkuU7Lc57efP2k4+yqbglKRscnWyQ1PbODT/dX8zzdmBtSIxBc5QxNYkJfBYx8d4Pn1xdx+/nifm6eo/ivQVM+viUgJMA94Q0Tetk4tAIpEZCvwEnCbMabGOnc78DiwD8c3ggF/sRegusmR911e39YnXaue+vQQMVERLD7rRHZPfk5arwT/ji47r20t44tTRnS7CzQjOZaRqXG9ttlrm/U+3mYXfeu8cWSmxPHl6Vm8sGzeafsCXF1ofRsINJ22vrWD6x7/jI1HavnD4jM9ZiWFyjVzRtPa0cWs0WkeO1Cp8BRots/LwMtujq8CVnl4TiEwLZD3DUfONf/Wji4aWjv93grvj/qWDlZvKuWK/JEnVUTMz0nj9aJyKhvaGJ4Suq30H+yuoqa53e2F3lPNyE7rtZl/UUk9IjDNi2UfgNSEaD75rwu9yqIZm55I7rAE3t9dxfXzcv0aX01zO9f/5TP2HG3k4WtncfEZmX69TiC+OHUEv7xiGl+aOiJk1xdU39B/zV7iDP4A5Q2927JwZWExrR1dLD0n96TjZ45OA2BziGf/qzeVMCwxhgUTe14ymJmTxuFjLdS1hH6HbFFJHeMzknyqv+JL+uT5k4bz6f5qv2qxVza2sXj5WvZVNvHYkoI+Cfzg+PteP3cMI0I4OVB9Q4N/L6lushEd6QgcvXnRt8tueHrtIeaMHcoZI0+e4Z4xMpWoCAnp0k9dSzv/3FnJ5fkjvZo5Oit8hnrpxxhDkYedvcFyweThtHXYWXfgmE/P67IblvxlPcU1rTx5w1mcP8lzJU6l/KXBv5dUN7YzKdOxQag30z3/ufMoJbWt3HjKrB8c3YKmZKWENOPn9aJy2rvsXNVNlo+raVYwLgrxt5GKhjaqGm3M8HLJxx9njx1KXHQEH/iY9bNm51F2VTTywFXTg75PQCknDf69pLrJxtSsFER6d+b/1KeHGJka57HnaX6OY409GI273Vm9qYRJI5I5Y6R3VR9T4qIZl5EY8pm/s2H8DCvlNRTioiM5d3w67+2q9Oki/2MfHSB7SHy3mVFKBUqDfy/o7LJT09JOZmo86UmxVPjRJs8fuysa+XT/Ma6fl+uxoUR+ThrN7V0hKVu892gjm47U8bVZo3wq/jWzFy76FpXUERUhTPWQqhks508ezpGaFg542RJy05FaCg/X8q3zxgbcBESp7uh/Xb2gpqUdYyAjKYas1DgqGnqnlPJTnx4k9pT0zlPlWxd9txTXBv39//T+PuKjI/mGj+mJM7JTqWy0hXR5rKiknokjkk8qKx0K51sXud/f5V3K5+MfHyAlLoqre7ngnhp8NPj3gupGR+ZKelIsmSlxvTLzr21u5+XNpXztzFEMSfRcumDssERS4qKCftH3YHUzr20t47q5o31uuD3DpcJnKBhjPJZxDracoQnkDU/yat3/8LFm/rG9gmvnjvGpBLNS/tDg3wuONTtm+unJsWSlxvXKmv+LhcW0ddhPS+88VUSEMDMnjc1Bvuj78Pv7iI6M4BY/CoCdMTKFqAhha4gu+hbXtFLX0nH8l0yoXTB5OJ8dPEazrfuaRU/86yCREcINPfybKRUMGvx7gTPHPz0plszUeBrbOmnqIRAEorPLzjNrDzN33FCP5QdcnTl6CHuONvYYnLxVXNPC6s2lXDNndLe7YD2Ji45kUmby8Yuyweb8RtEbM3+A8ydl0NFl+GRftcfH1LW0s7KwhEX5ozSnXvUKDf694MSyj2PNH0Kb7rlm51FK61q54Rzvin+dmZOG3RC0YPvwB/uJFOG2L4z3+zWcO31DUQpjW2k9MVERp9XmD5WCMUNJio3qttDbs58dobWji1vma6lk1Ts0+PeC6iYbsVERJMVGkdkLwf/JTw4xKi3eY3rnqWY6K3wGYZmltK6VlzYWc/VZ2cf/rv6YmZ1KQ1snh44FXhjtVFuL65iSlRKSRivuxERFcN6EdD7Y7T7l09bZxZOfHGLBxIzje0GUCjUN/r2gqslGelIsInJ85l8eoou+7Z121h+qYVH+SK9LEQxNjGHMsISgZPz8+cP9ANx+vncNwz0JVYVPu92wvbT++E7i3nLB5AzK69vY7aYX8Kuby6husrFMZ/2qF2nw7wXVTe2kJzkybpzruaEq7VxW14oxjsJivghGhc+jDW28sKGYq2ZlMyotPqDXmjgiibjoCK86e7V1dPHgu3vYV9lzk/UD1U00t3d5VcM/mJwlGt7fdfLSjzGGxz4+wJSsFM6dMKxXx6QGNw3+vaC60THzB8fFzCEJ0SHL+CmpdXyjyB6S4NPz8nPSONpgC+gbyZ8/PODoVRzgrB8gKjKCM0am9jjzt9sN331xCw/9cy9Ln9hAVWP3eyic1zVmhnBnrzsjUuKYmpVyWnevD/ZUsbeyiWULxvq0EU6pQGnw7wXVTSeCP0BmanzI1vyLax1r5DlDfZt5B9rZq6rRxrOfHeaK/FGMHubbLx5PZmansb2sns4uu8fH/OqNnby1vYIl88ZwrNnGsmcKu62iWVRST0JMJOMzkoIyRl9cMDmDjYdrqW/tOH7ssY8OkJkSx2UzRvb6eNTgpsE/xOx2w7HmdtKTT2y0CmWuf0ltC5ERQqaP6YJTR6YQExnh99LP4x8foKPLzh0X+J/hc6qZOam0ddjZc9R96YnHPz7AE58c5KZzx3Lfomn8/up8Nh+p44cvFXnMEioqqWPayFSfSjMHywWThtNlN/xrryPlc3tpPZ/uP8aN5+ZqrXzV6wLt5PVbEdklIkUi8rKIpLmcu1tE9onIbhG52OX4bBHZZp17SAb4d9261g667OaUmX8cFSFa8y+uaWVkWpzPdWFioyKZMjLFr9r+Nc3tPLPuMF+dOZJxQZxRd3fR942icu5/cydfnp7Jj78yBYBLp2fxg4sn8drWMv743r7TntPRZWdHWUOv5fefKj8njdT46ONLP499fICk2CiuOXt0n4xHDW6BTjfeBaYZY2YAe4C7AURkKrAYOAO4BHjY2dMXeARYhqOvb551fsBy3eDllJUSR01zu19NPnpSUttCdpp/yy5n5qSxraT7ZRZ3/vKvA7R2dHHnBYGv9bvKHZZASlzUaRU+1x+s4bsrtzB79BAevDqfCJdZ/LfPH8+VZ47iwXf38HpR2UnP23u0CVun3eu2jcEWFRnBgokZfLC7ipLaFl4vKmfxWTmkxPVeVzelnAIK/saYd4wxzm2h6wBnBa9FwAvGGJsx5iCOfr1zRCQLSDHGrDWO7+UrgCsCGUN/52zcfurMH0KT8VNc2+rzer9Tfk4arR1dHpdZ3Klv6eDpTw/z5WlZ5AV505SInNbWcV9lI7esKCR7SDyPLSk4rTCbiPDrq6Yze8wQvr9y60klIpyv403D9lC5YFIG1U02fvhSEQA3nufdRjylgi2YC403caIZ+yig2OVciXVslHX/1ONuicgyESkUkcKqKt8aYvQXVdbMP+OkNX9HcA72un9bRxdVjTafM32c8v3Y7PXEJwdpsnVy54XBnfU7zchOZXdFI20dXVQ2tLH0iQ1ER0bw9I1zPBasi42K5M/XzyYjOZZbVhQez2AqKq0nJS6KMUG6IO2PBRMzEIFP9x/jshlZAafEKuWvHoO/iKwRke1ubotcHnMv0Ak86zzk5qVMN8fdMsYsN8YUGGMKMjJ67v/aHx1rOlHR0ylUM39nmqe/M/8xwxIYkhDN5iPebfZqaOvgyU8O8qWpI7yqIeSPGdlpdNoNGw7VcNPTG6htaefJG84iZ2j3ATw9KZa/LD2LlvYubn66kJb2TopK6piRndanKZXpSbHHr2VoKQfVl3qsG2uMWdjdeRFZClwGXGROpFiUAK4FybOBMut4tpvjA5azd29q/Il13czju3yDHfwdaZ7+zvxFxKfNXis+PURDWyffuSjPr/fzhvPbyB3PbqK5vYvHlxR4vWY/KTOZP15zJt96egPfeX4zuysaubkfBNw7zh/P1pI6pvXyRjOlXAWa7XMJ8CPgcmOMaxGW14DFIhIrImNxXNhdb4wpBxpFZK6V5bMEeDWQMfR31U02hiXGnjTbTIqNIjk2Kui5/sXOmb+fwR8gP2cI+6qaaGzr6PZx6w/W8Kf393PR5OEhDWKZqXEMT46loa2T+6+YxgWTfWtmfsHk4dz7lams2VlJR5fp9bIO7nzpjEx+cPHkvh6GGuQC7Rjxf0As8K4V3NYZY24zxuwQkZXA5ziWg+4wxjhTW24HngLicVwjeOu0Vx1AqpvaGZZ0+tp0Zmpc0Ov7lNS2EBMZwfBk35qnuMofnYaxKnye66F5+MbDNdz45Hqy0uL49VXT/X4vb/37RXnY7YbFc/xLibzp3Fz2VzXx4oZi8nOGBHl0SoWngIK/McbjVT5jzP3A/W6OFwLTAnnfcHLq7l6nzNS4oM/8S2paGTUk/qTUR1/lW+vRW4rr3Ab/LcV1LH1iA8NT4nj+lrl+1ev31fVzxwT0fBHhV4umcccFEwKqNKrUQKLbCkPMta6Pq1Ds8i2pbSF7SGDZI6kJ0YxLT3Tb2WtbST3X/+UzhibG8NwtZ4dV05GICNHMGqVcaPAPIWOMo6Jnsrtln3iqmmx0+LihqjvFta1+X+x15bzo61oi4fOyBq77y2ekxEXz3C1nH09XVUqFJw3+IdTQ1kl7l50MDzN/Y6CyhyqU3mq2dVLT3B7wzB8c6/7VTTZK6xzXJHZXNHLdXz4jMSaSF5bNDcovGKVU3wr0gq/qhrvSDk6uHb2CsRxxIsc/ODN/cKzvt3V0ce3j64iOFJ67ZW5QXl8p1fc0+IeQu9IOTs6qm8G66Hsixz/wXySTMx0tDl/bUsYviusQEZ6/ZS65PjaIUUr1X7rsE0LVzt29btb8g93OsbjGquMfhCWZmKgIpo1M4Z3Pj2KM4flbzg5qtU6lVN/T4B9C3S37pMZHExcdEcSZfytx0RHH20UGauHUEQxPjuXZm+cyYbg2FVdqoNFlnxCqbrIRITAk4fSA7GjmHk95kOr7FNe2kD0kIWh1a25bMJ5bF4zvk6YnSqnQ0+AfQtVNNoYmxnoMoJkpwdvoVVLbGpT1fqdANooppfo/XfYJoeqm9m6XYbKCuMu3uKYlKOv9SqnBQYN/CHkq7eCUmRrH0YY27HaPVa29Ut/aQUNbZ1Bn/kqpgU2Dfwg5gn/3M/9Ou6G6ObCNXs40T83BV0p5S4N/CFU3tnc78x8RpFx/5wYvnfkrpbylwT9Emm2dtHZ0kd5NeeVgtXMMZo6/Umpw0OAfIt3l+Du5lngIREltK4kxkaQlRPf8YKWUQoN/yJwI/p7X/IclxhAdKQHP/EtqW8gZGrwcf6XUwBdoG8ffisguESkSkZdFJM06nisirSKyxbo96vKc2SKyTUT2ichDMkAjVlXj6Y3bTxURIYxIiaMiwBIPwc7xV0oNfIHO/N8FphljZgB7gLtdzu03xuRbt9tcjj8CLMPR1zcPuCTAMfRLzpl/Rg8tFQNt6mKMobimRcssK6V8ElDwN8a8Y4zptH5cB2R393gRyQJSjDFrjaNTyArgikDG0F85g//QxO5r7WSmxnM0gBIPdS0dNLd36cxfKeWTYK7538TJzdjHishmEflQROZbx0YBJS6PKbGOuSUiy0SkUEQKq6qqgjjU0KtusjEkIZroyO4/YufM37Vrli+KNcdfKeWHHmv7iMgaINPNqXuNMa9aj7kX6ASetc6VA6ONMcdEZDbwioicAbhb3/cY9Ywxy4HlAAUFBYFtg+1lPeX4O2WmxGHrtFPX0sGQHr4luKM5/kopf/QY/I0xC7s7LyJLgcuAi6ylHIwxNsBm3d8oIvuBiThm+q5LQ9lAmX9D7996Ku3glHm8rn+bX8HfmeOva/5KKV8Emu1zCfAj4HJjTIvL8QwRibTuj8NxYfeAMaYcaBSRuVaWzxLg1UDG0F8da25nmBe19Y/n+jf4l/FTUttKSlwUqfGa46+U8l6gJZ3/D4gF3rUyNtdZmT0LgPtEpBPoAm4zxtRYz7kdeAqIx3GN4K1TX3QgqG70buaf5TLz90exleOvlFK+CCj4G2MmeDi+Cljl4VwhMC2Q9+3v2jq6aLR19pjmCZCRFEuE+L/Lt6S2lfEZ2ltXKeUb3eEbAt7s7nWKioxgeLJ/df2NMY7dvbrer5TykQb/EDjeuN2LZR9wrPtX+JHrX93UTluHXTN9lFI+0+AfAtWNPRd1c+XvLl/N8VdK+UuDfwgcX/bxYs0frJm/H8H/RI6/Bn+llG80+IeAM/gP8zJvPzMljiZbJ41tHT69z4kcf132UUr5RoN/CFQ3tZMcF0VcdKRXj/e3rn9JbStDE2NIjA00Y1cpNdho8A+BqiYbGV6u94P/Hb0cmT4661dK+U6Dfwh4u8HLKSuAmb+u9yul/KHBPwSqm2xelXZwGp7i+EXhS7qn3W4orW0le6jO/JVSvtPgHwLVTd5V9HSKjYokPSnGp2WfykYb7V12nfkrpfyiwT/I2jvt1Ld2+BT8wZnu6X1xt+M5/rrmr5Tygwb/IKtptnb3JvtWnjkzJd6nmX9JrZZyVkr5T4N/kJ2o6+PrzD/WpzX/4hpt4qKU8p8G/yCr8jP4Z6XGU9fSQWt7l1ePL6ltISM51uu9BEop5UqDf5A56/r4kucPjl2+4H3GT3FNq876lVJ+C7ST1y9FpEhEtojIOyIy0uXc3SKyT0R2i8jFLsdni8g269xDVkevAeN4RU8f1/xPNHXx7qJvSZ2WclZK+S/Qmf9vjTEzjDH5wOvATwFEZCqwGDgDuAR42NnWEXgEWIajtWOedX7AqG6ykRATSUKMbyUXnCUejnox8+/sslNW16Yzf6WU3wIK/saYBpcfEwFj3V8EvGCMsRljDgL7gDkikgWkGGPWWs3eVwBXBDKG/sbbxu2nyvShnWNFQxtddqOlnJVSfgu4IpiI3I+jEXs9cIF1eBSwzuVhJdaxDuv+qcc9vfYyHN8SGD16dKBD7RWO4O/bkg9AQoyjCbs3JR4000cpFageZ/4iskZEtru5LQIwxtxrjMkBngXudD7NzUuZbo67ZYxZbowpMMYUZGRk9Py36QeqG33b3evK26YuJcc3eOnMXynlnx5n/saYhV6+1nPAG8DPcMzoc1zOZQNl1vFsN8cHjOomG7PGDPHrud42dSmpbUUEstLi/HofpZQKNNsnz+XHy4Fd1v3XgMUiEisiY3Fc2F1vjCkHGkVkrpXlswR4NZAx9CedXXZqWtrJ8GPZBxzpnt7M/ItrW8hMiSM2SnP8lVL+CXTN/wERmQTYgcPAbQDGmB0ishL4HOgE7jDGOHcv3Q48BcQDb1m3AaGmpR1jvG/feKrM1Diqm2y0d9qJifL8e9lRylnX+5VS/gso+Btjrurm3P3A/W6OFwLTAnnf/uqYM8ffzzX/kWmOgP7XdYe58dxcPG2BKKlpYe64Yf4NUiml0B2+QeVvXR+nL0/PYn5eOve9/jlLnlhPWd3pG77aO+1UNGiOv1IqMBr8g+hE8PdvzT8pNooVN83h/q9NY+PhWi7+349YtbEEx5YIh/L6VuwGsjXHXykVAA3+QVTd6Czt4N/MH0BEuPbsMfzjrgVMyUzh+3/byrJnNlJl1QwqqdUcf6VU4DT4B1F1k42YqAiSYwPeO8foYQk8v2wuP/7KFD7cU8WXfv8hb24rp7hGc/yVUoELPEqp46qabGQkxXq8UOuryAjh5vnjOH9SBt9buZVvP7uJjORYIiPkeCE4pZTyh878g8jRu9e/9f7uTBiezKrbz+F7X5xIbXM7OUPiiYrUfzqllP905h9E1Y22kM3IoyMj+M5FeXx5eiYdXR4rYiillFc0+AdRdZON6aNSQ/oeE4Ynh/T1lVKDg64dBIndbjjW3M6wECz7KKVUsGnwD5K61g667MbvDV5KKdWbNPgHyfENXgHk+CulVG/R4B8kzsbtocj2UUqpYNPgHyRV1sw/Q5d9lFJhQIN/kARa0VMppXqTBv8gqW6yERUhpMZH9/VQlFKqRxr8g6S6ycawpBgiIoJT2kEppUIp0DaOvxSRIhHZIiLviMhI63iuiLRax7eIyKMuz5ktIttEZJ+IPCTBKoTTxxylHXTJRykVHgKd+f/WGDPDGJMPvA781OXcfmNMvnW7zeX4I8AyHH1984BLAhxDv1BR38ZwTfNUSoWJgIK/MabB5cdEoNuiMyKSBaQYY9YaR4eSFcAVgYyhvyivbz3ehlEppfq7gNf8ReR+ESkGruXkmf9YEdksIh+KyHzr2CigxOUxJdYxT6+9TEQKRaSwqqoq0KGGTGt7F7UtHRr8lVJho8fgLyJrRGS7m9siAGPMvcaYHOBZ4E7raeXAaGPMmcD3gOdEJAVwt77v8duCMWa5MabAGFOQkZHh69+t15TVO7prjUzTGvtKqfDQY1VPY8xCL1/rOeAN4GfGGBtgs56/UUT2AxNxzPSzXZ6TDZT5NOJ+qLyuDYCsVJ35K6XCQ6DZPnkuP14O7LKOZ4hIpHV/HI4LuweMMeVAo4jMtbJ8lgCvBjKG/uD4zF+Dv1IqTARaz/8BEZkE2IHDgDOrZwFwn4h0Al3AbcaYGuvc7cBTQDzwlnULa2V1rYjAiFTN9lFKhYeAgr8x5ioPx1cBqzycKwSmBfK+/U15XRvpSbHERkX29VCUUsorusM3CMrqWxmpDdWVUmFEg38QlNVpjr9SKrxo8A+QMYby+jbN9FFKhRUN/gFqaO2kpb1Lc/yVUmFFg3+ASuucG7x05q+UCh8a/ANUbuX4Z+kFX6VUGNHgH6CyesfuXp35K6XCiQb/AJXVtRIdKdq7VykVVjT4B6i8rpURKXHawUspFVY0+AeorL5Na/oopcKOBv8AOTZ46cVepVR40eAfALvdcLShjSy92KuUCjMa/ANQ3WSjo8toXR+lVNjR4B8A3eCllApXGvwDUF6vHbyUUuEpKMFfRP5TRIyIpLscu1tE9onIbhG52OX4bBHZZp17yOroFZbK6rR3r1IqPAUc/EUkB/gicMTl2FRgMXAGcAnwsLOtI/AIsAxHa8c863xYKqtrIyEmktT46L4eilJK+SQYM//fAz8EjMuxRcALxhibMeYgsA+YIyJZQIoxZq0xxgArgCuCMIY+UV7fSlZqHGH85UUpNUgF2sD9cqDUGLP1lFOjgGKXn0usY6Os+6ceD0tl9W16sVcpFZZ67OErImuATDen7gXuAb7k7mlujplujnt672U4logYPXp0T0PtdWV1rUyeNLyvh6GUUj7rMfgbYxa6Oy4i04GxwFZr2SMb2CQic3DM6HNcHp4NlFnHs90c9/Tey4HlAAUFBR5/SfSF9k471U02svRir1IqDPm97GOM2WaMGW6MyTXG5OII7LOMMRXAa8BiEYkVkbE4LuyuN8aUA40iMtfK8lkCvBr4X6P3HW1owxi0ro9SKiz1OPP3hzFmh4isBD4HOoE7jDFd1unbgaeAeOAt6xZ2dIOXUiqcBS34W7N/15/vB+5387hCYFqw3revHO/gpcs+SqkwpDt8/VRWZ3Xw0mUfpVQY0uDvp7K6VoYkRBMfE9nzg5VSqp/R4O+n8vo2remjlApbGvz9pE1clFLhTIO/nxzBX2f+SqnwpMHfD822ThraOnXZRykVtjT4+8GZ5qnLPkqpcKXB3w+lzjRPXfZRSoUpDf5+KLd292Zp716lVJjS4O+Hsvo2RGBEigZ/pVR40uDvh7K6VkYkxxEdqR+fUio8afTyQ3l9q9b0UUqFNQ3+fiiva9OaPkqpsKbB30fGGEp1d69SKsxp8PdRbUsHtk67bvBSSoU1Df4+KqvTDV5KqfAXlOAvIv8pIkZE0q2fc0WkVUS2WLdHXR47W0S2icg+EXnIaucYNsq0g5dSagAIuJOXiOQAXwSOnHJqvzEm381THgGWAeuAN4FLCKNWjuX1jt29uuyjlApnwZj5/x74IWB6eqCIZAEpxpi1xhgDrACuCMIYek1ZfSsxkREMS4zp66EopZTfAgr+InI5UGqM2erm9FgR2SwiH4rIfOvYKKDE5TEl1jFPr79MRApFpLCqqiqQoQZNWV0bWWlxRESE1WqVUkqdpMdlHxFZA2S6OXUvcA/wJTfnyoHRxphjIjIbeEVEzgDcRUyP3xiMMcuB5QAFBQU9frPoDeV1rVrTRykV9noM/saYhe6Oi8h0YCyw1bpmmw1sEpE5xpgKwGY9f6OI7Acm4pjpZ7u8TDZQFtDfoJeV17dx9tihfT0MpZQKiN/LPsaYbcaY4caYXGNMLo7APssYUyEiGSISCSAi44A84IAxphxoFJG5VpbPEuDVwP8avaPLbqhoaNNMH6VU2As428eDBcB9ItIJdAG3GWNqrHO3A08B8TiyfMIm06eysY0uu9G6PkqpsBe04G/N/p33VwGrPDyuEJgWrPftTWXOJi6a5qmUCnO6w9cHusFLKTVQaPD3gbN3ry77KKXCnQZ/H5TVtZEUG0VKXHRfD0UppQKiwd8HZVrKWSk1QGjw90F5fZvW9FFKDQga/H1QXq8zf6XUwKDB30ttHV1UN7VrmqdSakDQ4O+lCmcpZ03zVEoNABr8vVRmpXmO1KJuSqkBQIO/l47v7tWZv1JqANDg76Vya3dvps78lVIDgAZ/L5XVtzEsMYa46Mi+HopSSgVMg7+XHBu8dMlHKTUwaPD3Unm9dvBSSg0cGvy9VF6nTVyUUgOHBn8vNLR10Gjr1N29SqkBI6DgLyI/F5FSEdli3b7scu5uEdknIrtF5GKX47NFZJt17iGrnWO/Vm6leWpdH6XUQBGMmf/vjTH51u1NABGZCiwGzgAuAR529vQFHgGW4ejrm2ed79eOb/DSmb9SaoAIVQ/fRcALxhgbcFBE9gFzROQQkGKMWQsgIiuAKwhhH9+bn97A4WMtAb1GQ1sHoDN/pdTAEYzgf6eILAEKge8bY2qBUcA6l8eUWMc6rPunHndLRJbh+JbA6NGj/Rrc6KGJxEQF/gVnZGq8ZvsopQaMHoO/iKwBMt2cuhfHEs4vAWP9+TvgJsDdOr7p5rhbxpjlwHKAgoICj4/rzk+/OtWfpyml1IDWY/A3xiz05oVE5DHgdevHEiDH5XQ2UGYdz3ZzXCmlVC8KNNsny+XHrwHbrfuvAYtFJFZExuK4sLveGFMONIrIXCvLZwnwaiBjUEop5btA1/x/IyL5OJZuDgG3AhhjdojISuBzoBO4wxjTZT3nduApIB7Hhd6QXexVSinlnhjj11J6rysoKDCFhYV9PQyllAorIrLRGFNw6nHd4auUUoOQBn+llBqENPgrpdQgpMFfKaUGobC54CsiVcBhP5+eDlQHcTihFE5jhfAabziNFcJrvOE0Vgiv8QY61jHGmIxTD4ZN8A+EiBS6u9rdH4XTWCG8xhtOY4XwGm84jRXCa7yhGqsu+yil1CCkwV8ppQahwRL8l/f1AHwQTmOF8BpvOI0Vwmu84TRWCK/xhmSsg2LNXyml1MkGy8xfKaWUCw3+Sik1CA3o4C8il1gN5PeJyH/19Xh6IiKHrOb2W0Sk31WxE5EnRKRSRLa7HBsqIu+KyF7rzyF9OUYnD2P9uYiUWp/vFhH5cl+O0UlEckTkfRHZKSI7ROQu63h//Ww9jbfffb4iEici60VkqzXWX1jH++tn62m8Qf9sB+yav9Uwfg/wRRxNZDYA1xhjPu/TgXXD6nFcYIzpl5tPRGQB0ASsMMZMs479Bqgxxjxg/YIdYoz5UV+O0xqXu7H+HGgyxvxPX47tVFZfjCxjzCYRSQY24uhtfQP987P1NN6r6Wefr9U3JNEY0yQi0cC/gLuAK+mfn62n8V5CkD/bgTzznwPsM8YcMMa0Ay/gaCyv/GSM+QioOeXwIuBp6/7TOIJAn/Mw1n7JGFNujNlk3W8EduLobd1fP1tP4+13jEOT9WO0dTP038/W03iDbiAH/1FAscvP3TaL7ycM8I6IbLSa14eDEVaHNqw/h/fxeHpyp4gUWctC/eKrvisRyQXOBD4jDD7bU8YL/fDzFZFIEdkCVALvGmP69WfrYbwQ5M92IAd/n5rF9xPnGmNmAZcCd1hLFyp4HgHGA/lAOfC7Ph3NKUQkCVgF/IcxpqGvx9MTN+Ptl5+vMabLGJOPo2f4HBGZ1sdD6paH8Qb9sx3Iwd9TE/l+yxhTZv1ZCbyMY+mqvzvq7OVs/VnZx+PxyBhz1Pofyw48Rj/6fK313VXAs8aY1dbhfvvZuhtvf/58AYwxdcAHONbP++1n6+Q63lB8tgM5+G8A8kRkrIjEAItxNJbvl0Qk0bp4hogkAl8Ctnf/rH7hNWCpdX8p8GofjqVbzv/ZLV+jn3y+1kW+vwA7jTEPupzql5+tp/H2x89XRDJEJM26Hw8sBHbRfz9bt+MNxWc7YLN9AKx0qP8FIoEnjDH39+2IPBORcThm+wBRwHP9bbwi8jxwPo4Ss0eBnwGvACuB0cAR4BvGmD6/0OphrOfj+NpsgEPArc51374kIucBHwPbALt1+B4c6+j98bP1NN5r6Gefr4jMwHFBNxLHZHelMeY+ERlG//xsPY33GYL82Q7o4K+UUsq9gbzso5RSygMN/kopNQhp8FdKqUFIg79SSg1CGvyVUmoQ0uCvlFKDkAZ/pZQahP4/okG0JWMkEG0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
